# Youtube statistics tool v0.1 
# author: Rene Pickhardt
# email: rene@rene-pickhardt.de
# date: november 9th 2011
# website: http://www.rene-pickhardt.de
# 
# This tool is supposed to help you discovering through which websites
# people came to watch your videos on youtube
#
# learn how it works at:
# visit my blog for instructions:
# http://www.rene-pickhardt.de/youtube-statistics-and-analytics-tool
# 
# or have a look in the readme file that comes along
#
# this software is open source!
#
# no warrenty
#
# feel free to use, redistribute and change it it as long as you
# mention the author. In latter cases I would appriciate if you'd drop
# me a quick message.
# if you are musician, label, booker I would also be happy to receive your feedback

import os

dir = "/home/rpickhardt/Downloads/example/"
fileToMerge = "merged.csv"
outDir = dir + "output/"
numberOfEmbeddedVideos = 40


# author: rene pickhardt
# date: November 9th 2011
#
# @param: directory = directory to start looking for subdirectories.
# returns a list of directories which came from unzipping your youtube statistics
def getDirs(directory):
	dirs = []
	for f in os.listdir(directory):
		if f.find(".")>1:
			continue
		d = dir+f
		dirs.append(d)
	return dirs

# author: rene pickhardt
# date: November 9th 2011
# returns a list of absolut paths to the files that have the youtube statistics
#
# @param: pattern to match should be one of the following:
# @param:
# "referrers"
# "views"
# "locations"
def getFiles(directories, pattern):
	files = []
	for d in directories:
		for f in os.listdir(d):
			if f.find(pattern)>0:
				files.append(d+"/"+f)
			
	files.sort()
	return files

# author: rene pickhardt
# date: November 9th 2011
# 
# merges all the files from your youtube video to one file that is to be processed
# @param files: list of filnames (including paths)
# @param out: name of the output file
def MergeFiles(files,out):
	out = open(out,"w");
	for f in files:
		stream = open(f,"r")
		stream = stream.read()
		lines = stream.split("\n")
		for line in lines:
			if line.find("Date,Region,VideoID,Title,Source type,Detail,Referred views") > 0:
				continue
			out.write(line+"\n")
	out.close()

# author: rene pickhardt
# date: November 9th 2011
# 
# retrieves all different details from youtube as a dictionary
#
# @return: dictionary with Youtube Details as keys and counts as values.

def retrieveYTDetails(file):
	stream = open(file,"r")
	stream = stream.read()
	lines = stream.split("\n")
	d = {}
	for line in lines:
		values = line.split(',')
		if len(values) > 5:
			if values[4] in d:
				d[values[4]] = d[values[4]]+1
			else:
				d[values[4]]=1
	return d


# author: rene pickhardt
# date: November 9th 2011
# 
# aggregates all reffered views for a certain Youtube Detail attribute
#
# @param detail: the detail level of youtube
# @return: dictionary with referrers as keys and counts as values.
#
# IMPROVE EXCHANGE DETAIL TO A LIST AND ALLOW DIFFERENT DTAIL LEVELS AT A TIME

def aggregate(detail, file):
	stream = open(file,"r")
	stream = stream.read()
	lines = stream.split("\n")
	d = {}
	for line in lines:
		values = line.split(',')
		if len(values) == 7:
			if values[4]<>detail:
				continue;
			tmp = values[5]
			tmp = tmp.replace("www.","")
			if tmp in d:
				if detail <> "Source type":
					d[tmp] = d[tmp]+int(values[6])
				else:
					d[tmp] = d[tmp] + 1
			else:
				d[tmp]=1
	return d

# author: rene pickhardt
# date: November 9th 2011
# 
# transfers a dictionary with urls / search queries as keys and their counts as values
# to a dictionary with counts as keys and lists of urls as values
#
# @return: dictionary with counts as keys and urls / search queries values.

def AggregateSameNumberOfViews(dict):
	result = {}
	for key, values in dict.iteritems():
		if values in result:
			result[values].append(key)
		else:
			result[values] = [key]
	return result

def CreateEmbeddedRelatedVideosHtmlFile(result,k):
	keys = result.keys()
	keys.sort()
	keys.reverse()
	w = open(outDir+"relvideos.html","w")
	cnt = 0;	
	for key in keys:
		for el in result[key]:
			cnt = cnt +1
			w.write(str(key) + " views after the following video has been watched in \t" + "<iframe width=\"560\" height=\"315\" src=\"http://www.youtube.com/embed/"+el+"?hd=1\" frameborder=\"0\" allowfullscreen></iframe><br><br>" + "\r\n")
		if cnt > k:
			break
	w.close()

# author: rene pickhardt
# date: November 9th 2011
# 
# handels everything that is to do in order to retrieve youtube refferer and other stats

def retrieveOrderedListsForYTDetails():
	directories = getDirs(dir)
	files = getFiles(directories,"referrers")
	MergeFiles(files,fileToMerge)
	d = retrieveYTDetails(fileToMerge)

	for k in d.keys():
		print "processing " + k + " ... " + str(d[k]) + " items."
		w = open(outDir+k+".txt","w")
		dict = aggregate(k,fileToMerge)
		result = AggregateSameNumberOfViews(dict)
		if k == "RELATED_VIDEO":
			CreateEmbeddedRelatedVideosHtmlFile(result,numberOfEmbeddedVideos)			

		keys = result.keys()
		keys.sort()
		keys.reverse()
		for key in keys:
			for el in result[key]:
#				print str(key) + "  " + el
				w.write(str(key) + "\t" + el + "\r\n")
		w.close()

retrieveOrderedListsForYTDetails()
